version: 2.1 # Adds support for executors, parameterized jobs, etc
orbs:
  gh: circleci/github-cli@2.0
parameters:
  # These parameters are not meant to be changed they are more constants for the build change these in mk/dev.mk
  go_version:
    type: string
    default: "1.22.0"
  first_k8s_version:
    type: string
    default: "v1.23.17-k3s1"
  last_k8s_version:
    type: string
    default: "v1.29.1-k3s2"
  ubuntu_image:
    type: string
    default: "ubuntu-2204:2022.10.2"
  gh_action_build_artifact_name:
    type: string
    default: ""
  gh_action_run_id:
    type: string
    default: ""
  e2e_param_k8sVersion:
    type: string
    default: "v1.28.1-k3s1"
  e2e_param_arch:
    type: string
    default: "amd64"
  e2e_param_parallelism:
    type: integer
    default: 1
  e2e_param_target:
    type: string
    default: ""
  e2e_param_cniNetworkPlugin:
    type: string
    default: flannel
# See https://circleci.com/docs/2.0/configuration-reference/#commands-requires-version-21.
commands:
  install_build_tools:
    description: "Install an upstream Go release to $HOME/go"
    parameters:
      go_arch:
        type: string
        default: amd64
    steps:
      - run:
          # `unzip`    is necessary to install `protoc`
          # `gcc`      is necessary to run `go test -race`
          # `git`      is necessary because the CircleCI version is different somehow ¯\_(ツ)_/¯
          # `xz-utils` is necessary to decompress xz files
          name: "Install basic tools"
          command: |
            if [ -r /etc/os-release ]; then source /etc/os-release; fi
            case "$ID" in
            ubuntu)
              if ! command -v sudo 2>&1 >/dev/null; then
                apt update
                apt install -y sudo
              fi

              sudo apt update
              sudo env DEBIAN_FRONTEND=noninteractive apt install -y curl git make unzip gcc xz-utils
              ;;
            esac
      - run:
          name: "Install Go"
          # See https://golang.org/doc/install#tarball
          command: |
            curl -s --fail --location https://dl.google.com/go/go<<pipeline.parameters.go_version>>.linux-<<parameters.go_arch>>.tar.gz | tar -xz -C $HOME
            echo 'export PATH=$HOME/go/bin:$PATH' >> $BASH_ENV
            # if GOPATH is not set, `golang-ci` fails with an obscure message
            # "ERROR Running error: context loading failed: failed to load program with go/packages: could not determine GOARCH and Go compiler"
            echo 'export GOPATH=$HOME/go' >> $BASH_ENV
  halt_non_priority_job:
    description: "don't run following steps if in PR and doesn't have a specific label"
    parameters:
      label:
        type: string
        description: a label to for running this (if empty no label override is allowed)
        default: "ci/run-full-matrix"
    steps:
      - run:
          name: maybe-halt
          command: |
            if [[ "<< pipeline.git.branch >>" == "master" || "<< pipeline.git.branch >>" == "release-"*  ]]; then
              echo "on a main branch so don't halt job"
              exit 0
            fi
            if [[ "<< pipeline.git.tag >>" != "" ]]; then
              echo "on a tag to don't halt job"
              exit 0
            fi
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
              echo "<<parameters.label>> label present on PR so don't halt job"
              exit 0
            fi
            echo "halt running job"
            circleci-agent step halt
            exit 0
  setenv_depending_on_priority:
    description: "don't run following steps if in PR and doesn't have a specific label"
    parameters:
      label:
        type: string
        description: a label to for running this (if empty no label override is allowed)
      env:
        type: string
        description: a set of env var to set if it's a priority job
    steps:
      - run:
          name: maybe-add-env
          command: |
            function add_env() {
              echo 'export << parameters.env >>' >> "$BASH_ENV"
              exit 0
            }
            if [[ "<< pipeline.git.branch >>" == "master" || "<< pipeline.git.branch >>" == "release-"*  ]]; then
              echo "on a main branch so run everything"
              add_env
            fi
            if [[ "<< pipeline.git.tag >>" != "" ]]; then
              echo "on a tag so run everything"
              add_env
            fi
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
              echo "<<parameters.label>> label present on PR so run everything"
              add_env
            fi
            exit 0
  halt_job_if_labeled:
    description: "don't run following steps if PR has a specific label"
    parameters:
      label:
        type: string
        description: a label to for running this (if empty no label override is allowed)
        default: ""
    steps:
      - run:
          name: maybe-halt
          command: |
            if [[ "<<parameters.label>>" != "" && $(${KUMA_DIR}/tools/ci/has_label.sh "<<parameters.label>>") == "true" ]]; then
              echo "Not running as the PR has label: <<parameters.label>>"
              circleci-agent step halt
              exit 0
            fi
            echo "PR doesn't have label <<parameters.label>> keep running job"
            exit 0
executors:
  golang-amd64:
    resource_class: xlarge
    docker:
      - image: "cimg/go:<< pipeline.parameters.go_version >>"
    environment:
      KUMA_DIR: .
      GO_VERSION: << pipeline.parameters.go_version >>
  golang-arm64:
    resource_class: arm.xlarge
    docker:
      - image: "cimg/go:<< pipeline.parameters.go_version >>"
    environment:
      KUMA_DIR: .
      GO_VERSION: << pipeline.parameters.go_version >>
  vm-xlarge-amd64:
    resource_class: xlarge
    machine:
      image: << pipeline.parameters.ubuntu_image >>
    environment:
      KUMA_DIR: .
      GO_VERSION: << pipeline.parameters.go_version >>
  vm-amd64:
    resource_class: large
    machine:
      image: << pipeline.parameters.ubuntu_image >>
    environment:
      KUMA_DIR: .
      GO_VERSION: << pipeline.parameters.go_version >>
  vm-arm64:
    resource_class: arm.large
    machine:
      image: << pipeline.parameters.ubuntu_image >>
    environment:
      KUMA_DIR: .
      GO_VERSION: << pipeline.parameters.go_version >>
jobs:
  go_cache:
    executor: golang-<< parameters.arch >>
    parameters:
      arch:
        description: the executor to run on
        type: string
        default: golang
    steps:
      - checkout
      - restore_cache:
          key: vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      - run:
          command: make dev/tools
      - run:
          name: "Download Go modules"
          command: |
            go mod download -x
      - save_cache:
          key: vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
          paths:
            - "/home/circleci/go/pkg/mod"
            - "/home/circleci/.kuma-dev"
  test:
    parameters:
      target:
        description: The test make target.
        type: string
        default: test
      arch:
        description: The golang arch.
        type: string
        default: amd64
    executor:
      name: vm-<< parameters.arch >>
    steps:
      - checkout
      - when:
          condition: {equal: [arm64, << parameters.arch >>]}
          steps:
            - halt_non_priority_job
      - halt_job_if_labeled:
          label: "ci/skip-test"
      - install_build_tools:
          go_arch: << parameters.arch >>
      - restore_cache:
          keys:
            - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      - run:
          name: "Run tests"
          command: |
            make << parameters.target >> TEST_REPORTS=1
      - store_artifacts:
          path: build/reports
          destination: /reports
      - store_test_results:
          path: build/reports
  e2e:
    parameters:
      k8sVersion:
        description: version of k3s to use or "kind" and "kindIpv6"
        type: string
        default: << pipeline.parameters.last_k8s_version >>
      parallelism:
        description: level of parallelization
        type: integer
        default: 1
      target:
        description: makefile target without test/e2e prefix
        type: string
        default: ""
      arch:
        description: The golang arch
        type: string
        default: amd64
      cniNetworkPlugin:
        description: The CNI networking plugin to use [flannel | calico]
        type: string
        default: flannel
    executor:
      name: vm-<< parameters.arch >>
    parallelism: << parameters.parallelism >>
    steps:
      - checkout
      - halt_job_if_labeled:
          label: "ci/skip-test"
      - halt_job_if_labeled:
          label: "ci/skip-e2e-test"
      - when:
          condition:
            or:
              - {equal: ["", << parameters.target >>]}
              - {equal: [calico, << parameters.cniNetworkPlugin >>]}
              - {equal: [kindIpv6, << parameters.k8sVersion >>]}
              - {equal: [arm64, << parameters.arch >>]}
              - {equal: [<< pipeline.parameters.first_k8s_version >>, << parameters.k8sVersion >>]}
          steps:
            - halt_non_priority_job
      - run:
          # This works around circleci limitation by skipping tests for combinations that don't make sense
          # See: https://discuss.circleci.com/t/matrix-exclude-entire-subset/43879
          name: skip_invalid_parameter_combinations
          command: |
            echo "Running with: \
              k8s:<< parameters.k8sVersion >> \
              target:<< parameters.target >> \
              parallelism:<< parameters.parallelism >> \
              arch:<< parameters.arch >> \
              cniNetworkPlugin:<< parameters.cniNetworkPlugin >> \
            "
            function skip() {
              echo "Non valid job combination halting job reason: $1"
              circleci-agent step halt
              exit 0
            }

            # Handle invalid test combinations
            if [[ "<< parameters.k8sVersion >>" == "kind" && "<< parameters.target >>" != "universal" ]]; then
              skip "kind should only be used when testing ipv6 or with e2e-universal"
            fi
            if [[ "<< parameters.k8sVersion >>" != kind* && "<< parameters.target >>" == "universal" ]]; then
              skip "universal only runs on kind"
            fi
            echo "Continuing tests"
      - install_build_tools:
          go_arch: << parameters.arch >>
      - restore_cache:
          keys:
            - vm-<< parameters.arch >>_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      # Mount files from the upstream jobs
      - attach_workspace:
          at: build
      - run:
          name: "Setup Helm"
          command: |
            make dev/set-kuma-helm-repo
      - when: # CircleCI's DNS on IPV6 prevents resolving inside Kind. When change to 8.8.8.8 and remove "search" section (. removes it), resolving works again
          condition:
            equal: [<< parameters.k8sVersion >>, "kindIpv6"]
          steps:
            - run:
                name: Enable IPV6 and change DNS
                command: |
                  cat \<<'EOF' | sudo tee /etc/docker/daemon.json
                  {
                    "ipv6": true,
                    "fixed-cidr-v6": "2001:db8:1::/64",
                    "dns": ["8.8.8.8"],
                    "dns-search": ["."]
                  }
                  EOF
                  sudo service docker restart
      - run:
          name: "Run E2E tests"
          command: |
            if [[ "<< parameters.k8sVersion >>" == "kindIpv6" ]]; then
              export IPV6=true
              export K8S_CLUSTER_TOOL=kind
              export KUMA_DEFAULT_RETRIES=60
              export KUMA_DEFAULT_TIMEOUT="6s"
            fi
            if [[ "<< parameters.k8sVersion >>" != "kind"* ]]; then
              export CI_K3S_VERSION=<< parameters.k8sVersion >>
              export K3D_NETWORK_CNI=<< parameters.cniNetworkPlugin >>
            fi
            if [[ "<< parameters.arch >>" == "arm64" ]]; then
              export MAKE_PARAMETERS="-j1"
            else
              export MAKE_PARAMETERS="-j2"
            fi

            if [[ "<< parameters.target >>" == "" ]]; then
              export GINKGO_E2E_LABEL_FILTERS="job-$CIRCLE_NODE_INDEX"
            fi
            env
            if [[ "<< parameters.target >>" != "" ]]; then
              target="test/e2e-<< parameters.target >>"
            else
              target="test/e2e"
            fi
            make ${MAKE_PARAMETERS} CI=true "${target}"
      - store_test_results:
          path: build/reports
      - store_artifacts:
          name: "Store logs"
          path: /tmp/e2e
  e2e_testing:
    parameters:
      k8sVersion:
        description: version of k3s to use or "kind" and "kindIpv6"
        type: string
        default: << pipeline.parameters.last_k8s_version >>
      parallelism:
        description: level of parallelization
        type: integer
        default: 1
      target:
        description: makefile target without test/e2e prefix
        type: string
        default: ""
      arch:
        description: The golang arch
        type: string
        default: amd64
      cniNetworkPlugin:
        description: The CNI networking plugin to use [flannel | calico]
        type: string
        default: flannel
    executor:
      name: vm-<< parameters.arch >>
    parallelism: << parameters.parallelism >>
    steps:
      - checkout
      - install_build_tools:
          go_arch: << parameters.arch >>
      - run:
          command: make dev/tools
      - run:
          name: "Download Go modules"
          command: |
            go mod download -x
      - run:
          name: "Build"
          command: |
            make build
      - run:
          name: "Distributions"
          command: |
            make -j build/distributions
      - run:
          name: "Images"
          command: |
            make -j images
            make -j docker/save
      - run:
          name: "Setup Helm"
          command: |
            make dev/set-kuma-helm-repo
      - when: # CircleCI's DNS on IPV6 prevents resolving inside Kind. When change to 8.8.8.8 and remove "search" section (. removes it), resolving works again
          condition:
            equal: [<< parameters.k8sVersion >>, "kindIpv6"]
          steps:
            - run:
                name: Enable IPV6 and change DNS
                command: |
                  cat \<<'EOF' | sudo tee /etc/docker/daemon.json
                  {
                    "ipv6": true,
                    "fixed-cidr-v6": "2001:db8:1::/64",
                    "dns": ["8.8.8.8"],
                    "dns-search": ["."]
                  }
                  EOF
                  sudo service docker restart
      - run:
          name: "Run E2E tests"
          command: |
            if [[ "<< parameters.k8sVersion >>" == "kindIpv6" ]]; then
              export IPV6=true
              export K8S_CLUSTER_TOOL=kind
              export KUMA_DEFAULT_RETRIES=60
              export KUMA_DEFAULT_TIMEOUT="6s"
            fi
            if [[ "<< parameters.k8sVersion >>" != "kind"* ]]; then
              export CI_K3S_VERSION=<< parameters.k8sVersion >>
              export K3D_NETWORK_CNI=<< parameters.cniNetworkPlugin >>
            fi
            if [[ "<< parameters.arch >>" == "arm64" ]]; then
              export MAKE_PARAMETERS="-j1"
            else
              export MAKE_PARAMETERS="-j2"
            fi

            if [[ "<< parameters.target >>" == "" ]]; then
              export GINKGO_E2E_LABEL_FILTERS="job-$CIRCLE_NODE_INDEX"
            fi
            env
            if [[ "<< parameters.target >>" != "" ]]; then
              target="test/e2e-<< parameters.target >>"
            else
              target="test/e2e"
            fi
            make ${MAKE_PARAMETERS} CI=true "${target}"
  build:
    executor: vm-xlarge-amd64
    steps:
      - run: # required to build arm64 images
          command: |
            sudo apt-get update
            sudo apt-get install -y qemu-user-static binfmt-support
      - install_build_tools
      - checkout
      - setenv_depending_on_priority:
          label: "ci/run-full-matrix"
          env: ENABLED_GOARCHES="arm64 amd64" ENABLED_GOOSES="linux darwin"
      - restore_cache:
          keys:
            - build_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      - run:
          command: make dev/tools
      - run:
          command: make clean
      - run:
          command: make check
      - run:
          command: make build
      - run:
          command: make -j build/distributions
      - run:
          command: make -j images
      - run:
          command: make -j docker/save
      - save_cache:
          key: build_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
          paths:
            - "/home/circleci/go/pkg/mod"
            - "/home/circleci/.kuma-dev"
      - persist_to_workspace:
          root: build
          paths:
            - distributions/out
            - docker
            - artifacts-linux-amd64
            - artifacts-linux-arm64
            - ebpf-amd64
            - ebpf-arm64
  distributions:
    executor: vm-amd64
    steps:
      - install_build_tools
      - checkout
      - setenv_depending_on_priority:
          label: ci/force-publish
          env: ALLOW_PUSH=true
      - setenv_depending_on_priority:
          label: ci/run-full-matrix
          env: ENABLED_GOARCHES="arm64 amd64" ENABLED_GOOSES="linux darwin"
      # Mount files from the upstream jobs
      - restore_cache:
          keys:
            - vm-amd64_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      - attach_workspace:
          at: build
      - run:
          name: inspect created tars
          command: for i in build/distributions/out/*.tar.gz; do echo $i; tar -tvf $i; done
      - run:
          name: Publish distributions to Pulp
          command: make publish/pulp
      - run:
          name: load images
          command: make docker/load
      - run:
          name: publish images
          command: |
            make docker/login
            # ensure we always logout
            function on_exit() {
              make docker/logout
            }
            trap on_exit EXIT
            make docker/push
            make docker/manifest
  container-structure:
    executor: vm-amd64
    steps:
      - checkout
      - halt_job_if_labeled:
          label: "ci/skip-container-structure-test"
      - halt_job_if_labeled:
          label: "ci/skip-test"
      - setenv_depending_on_priority:
          label: "ci/run-full-matrix"
          env: ENABLED_GOARCHES="arm64 amd64"
      - run:
          command: sudo apt-get update; sudo apt-get install -y qemu-user-static binfmt-support
      - restore_cache:
          key: vm-amd64_go.mod_{{ checksum "go.sum" }}_{{ checksum "mk/dependencies/deps.lock" }}_{{ checksum ".circleci/config.yml" }}
      - attach_workspace:
          at: build
      - run:
          command: make test/container-structure
workflows:
  version: 2
  kuma-commit:
    when: false # Stop running for the moment (likely we'll remove this in the future.
    # equal: ["", << pipeline.parameters.gh_action_build_artifact_name >>]
    jobs:
      - go_cache:
          name: go_cache-<< matrix.arch >>
          matrix:
            alias: go_cache
            parameters:
              arch: [amd64, arm64]
      - build:
          name: build
      - test:
          name: test-<< matrix.arch >>
          matrix:
            alias: test
            parameters:
              arch: [amd64, arm64]
          requires: [build, go_cache-<< matrix.arch >>]
      - e2e:
          name: legacy-k8s:<< matrix.arch >>-<< matrix.k8sVersion >>
          matrix:
            alias: legacy
            parameters:
              k8sVersion: [<< pipeline.parameters.first_k8s_version >>, << pipeline.parameters.last_k8s_version >>, kind, kindIpv6]
              arch: [amd64, arm64]
          parallelism: 3
          target: ""
          requires: [build, go_cache-<< matrix.arch >>]
      - e2e:
          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>
          matrix:
            alias: e2e
            parameters:
              k8sVersion: [<< pipeline.parameters.first_k8s_version >>, << pipeline.parameters.last_k8s_version >>, kind, kindIpv6]
              target: [kubernetes, universal, multizone]
              arch: [amd64, arm64]
          requires: [build, go_cache-<< matrix.arch >>]
      - e2e:
          name: << matrix.target >>:<< matrix.arch >>-<< matrix.k8sVersion >>-calico
          matrix:
            alias: calico
            parameters:
              k8sVersion: [<< pipeline.parameters.last_k8s_version >>]
              target: [multizone]
              arch: [amd64]
              cniNetworkPlugin: [calico]
          requires: [build, go_cache-amd64]
      - container-structure:
          name: container-structure
          requires: [build]
      - distributions:
          requires: [test, container-structure]
  manual-e2e:
    when:
      not:
        equal: ["", << pipeline.parameters.gh_action_build_artifact_name >>]
    jobs:
      - e2e_testing:
          filters:
            tags:
              only: /.*/
          k8sVersion: << pipeline.parameters.e2e_param_k8sVersion >>
          target: << pipeline.parameters.e2e_param_target >>
          arch: << pipeline.parameters.e2e_param_arch >>
          cniNetworkPlugin: << pipeline.parameters.e2e_param_cniNetworkPlugin >>
          parallelism: << pipeline.parameters.e2e_param_parallelism >>
